package com.sunlee.admin.api.module.job.template;

import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.sunlee.admin.api.module.job.dto.CrawlerResourceTaskDTO;
import com.sunlee.admin.api.module.job.template.context.CrawResourceContext;
import com.sunlee.admin.api.util.FileUtil;
import com.sunlee.common.core.exception.BusinessException;
import com.sunlee.common.core.utils.DateUtils;
import com.sunlee.common.core.utils.TimeUtils;
import com.sunlee.common.service.entity.ThirdCategoryEntity;
import com.sunlee.common.service.entity.ThirdProductCategoryMappingEntity;
import com.sunlee.common.service.entity.ThirdProductEntity;
import com.sunlee.common.service.enums.StatusEnum;
import com.sunlee.common.service.service.ThirdCategoryService;
import com.sunlee.common.service.service.ThirdProductCategoryMappingService;
import com.sunlee.common.service.service.ThirdProductService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.RestTemplate;

import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * 爬取第三方资源模板类
 *
 * @author sunlee
 */
@Slf4j
@Component
public abstract class CrawResourceTemplate {

    @Autowired
    private RestTemplate restTemplate;

    @Autowired
    private ThirdCategoryService thirdCategoryService;

    @Autowired
    private ThirdProductService thirdProductService;

    @Autowired
    private ThirdProductCategoryMappingService thirdProductCategoryService;

    public final void process(CrawResourceContext context) {
        // 流程开始处理前，进行初始化操作（推断所需爬取的二级分类）
        init(context);

        // 开始处理
        doProcess(context);

        // 流程处理结束后，进行收尾操作
        end(context);
    }

    /**
     * 主操作流程：
     * 1、根据初始化结果，遍历分类
     * 2、根据分类结果，遍历分页
     * 3、批量处理每页的资源
     *
     * @param context 上下文
     */
    private void doProcess(CrawResourceContext context) {

        int curCategoryIndex = 0;
        int categoryTotal = context.getThirdResourceCategoryMap().size();

        // 遍历二级（叶子）分类地址
        for (Map.Entry<String, Long> thirdCategoryEntity : context.getThirdResourceCategoryMap().entrySet()) {
            log.info("1、==========爬取资源：开始当前分类 {}/{}，tCid={}，id={}", ++curCategoryIndex, categoryTotal, thirdCategoryEntity.getKey(), thirdCategoryEntity.getValue());
            context.setCurThirdCategory(thirdCategoryEntity.getKey());

            // 单次循环初始化
            initLoop(context);

            if (context.isSkipProcess()) {
                continue;
            }

            // 遍历具体资源列表，资源收集、筛选、构造、下载、入库
            int resUrlTotal = context.getUrlList().size();
            int resUrlIndex = 0;
            for (String url : context.getUrlList()) {
                log.info("2、=====爬取资源：开始-当前资源第 {}/{} 页，url={}", ++resUrlIndex, resUrlTotal, url);
                context.setCurUrl(url);
                handlerResSaveProcess(context);
                log.info("2、=====爬取资源：结束-当前资源第 {}/{} 页", resUrlIndex, resUrlTotal);
            }

            // 单次循环结束操作
            endLoop(context);
            log.info("1、==========爬取资源：结束当前分类：tCid={},id={}", thirdCategoryEntity.getKey(), thirdCategoryEntity.getValue());
        }
    }

    /**
     * 遍历具体资源列表，资源收集、筛选、构造、下载、入库
     *
     * @param context 上下文
     */
    private void handlerResSaveProcess(CrawResourceContext context) {

        context.setSkipProcess(false);

        // 根据url，爬取所有第三方资源ids
        collectThirdResIds(context);

        if (context.isSkipProcess()) {
            return;
        }

        // 根据第三方资源ids，去掉不需要下载的资源ids
        // 注意：哪怕没有需新增的资源，也不能在此处跳出流程，因为后面的旧资源需走完分类映射流程
        filterThirdResIds(context);

        // 为已存在的第三方资源更新分类映射
        refreshResCategoryMapping(context);

        if (context.isSkipProcess()) {
            return;
        }

        // 构建资源下载链接
        collectDownloadUrls(context);

        if (context.isSkipProcess()) {
            return;
        }

        // 封装资源、分类实体
        buildEntityList(context);

        // 根据资源下载链接进行下载
        downloadRes(context);

        // 新增记录
        save(context);
    }

    private void refreshResCategoryMapping(CrawResourceContext context) {
        List<String> oldThirdResIdList = context.getOldThirdResIdList();

        if (CollectionUtils.isEmpty(oldThirdResIdList)) {
            log.info("无旧资源，故无需更新分类，新资源total={},ids={} ", context.getThirdResIdList().size(), context.getThirdResIdList());
            return;
        }

        CrawlerResourceTaskDTO dto = context.getDto();
        Long curThirdCategoryKeyId = context.getThirdResourceCategoryMap().get(context.getCurThirdCategory());

        // 根据第三方商品Id，查询实体记录
        List<ThirdProductEntity> oldAllThirdProductList = thirdProductService.list(new LambdaQueryWrapper<ThirdProductEntity>()
                .eq(ThirdProductEntity::getSource, dto.getSource())
                .in(ThirdProductEntity::getThirdResId, oldThirdResIdList));

        if (oldThirdResIdList.size() != oldAllThirdProductList.size()) {
            throw new BusinessException("更新分类时，数据库存在的第三方资源对比异常");
        }

        // 根据实体记录查询，收集主键ID
        List<Long> oldAllThirdProductIdList = oldAllThirdProductList.stream().map(ThirdProductEntity::getId).collect(Collectors.toList());

        // 根据第三方商品主键 及 第三方分类主键，查询分类关联数据
        List<ThirdProductCategoryMappingEntity> oldAllMappingList = thirdProductCategoryService.list(new LambdaQueryWrapper<ThirdProductCategoryMappingEntity>()
                .eq(ThirdProductCategoryMappingEntity::getCategoryId, curThirdCategoryKeyId)
                .in(ThirdProductCategoryMappingEntity::getProductId, oldAllThirdProductIdList)
        );

        if (CollectionUtils.isNotEmpty(oldAllMappingList)) {
            // 剔除已存在的商品Id
            List<Long> oldExistMappingPid = oldAllMappingList.stream().map(ThirdProductCategoryMappingEntity::getProductId).collect(Collectors.toList());
            oldAllThirdProductIdList.removeAll(oldExistMappingPid);
        }

        if (CollectionUtils.isEmpty(oldAllThirdProductIdList)) {
            log.info("旧资源已存在关联分类 {}，无需再次更新分类 {} ", context.getCurThirdCategory(), oldThirdResIdList);
            return;
        }

        // 封装商品分类映射实体
        List<ThirdProductCategoryMappingEntity> thirdProductCategoryMappingEntityList = oldAllThirdProductIdList.stream()
                .map(pKey -> {
                    ThirdProductCategoryMappingEntity mappingEntity = new ThirdProductCategoryMappingEntity();
                    mappingEntity.setProductId(pKey);
                    mappingEntity.setCategoryId(curThirdCategoryKeyId);
                    mappingEntity.setSource(dto.getSource());
                    return mappingEntity;
                }).collect(Collectors.toList());

        // 入库
        boolean mappingSaveResult = thirdProductCategoryService.saveBatch(thirdProductCategoryMappingEntityList);

        if (!mappingSaveResult) {
            throw new BusinessException("旧资源更新分类失败");
        }

        log.info("旧资源更新分类成功 {} ", oldAllThirdProductIdList);
    }

    /**
     * 根据url，爬取所有第三方资源ids
     *
     * @param context 上下文
     */
    protected abstract void collectThirdResIds(CrawResourceContext context);

    /**
     * 构建下载链接
     *
     * @param context 上下文
     */
    protected abstract void collectDownloadUrls(CrawResourceContext context);

    /**
     * 根据第三方资源ids，过滤不需要下载的资源ids
     *
     * @param context 上下文
     */
    protected void filterThirdResIds(CrawResourceContext context) {

        List<String> newThirdResIdList = context.getThirdResIdList();

        // 查询已有第三方商品
        List<ThirdProductEntity> existResList = thirdProductService.list(
                new LambdaQueryWrapper<ThirdProductEntity>()
                        .select(ThirdProductEntity::getId, ThirdProductEntity::getThirdResId, ThirdProductEntity::getStatus)
                        // 限定PPT来源
                        .eq(ThirdProductEntity::getSource, context.getDto().getSource())
                        // 根据第三方商品Id查
                        .in(ThirdProductEntity::getThirdResId, newThirdResIdList));

        // 收集已存在的第三方Id
        List<String> oldThirdResIdList = existResList.stream().map(ThirdProductEntity::getThirdResId).collect(Collectors.toList());
        // 设置已存在的第三方资源主键Id
        context.setOldThirdResIdList(oldThirdResIdList);

        // 去掉已存在的
        newThirdResIdList.removeAll(oldThirdResIdList);
        log.info("待新增资源 {} ", newThirdResIdList);
    }

    /**
     * 封装实体
     *
     * @param context 上下文
     */
    protected abstract void buildEntityList(CrawResourceContext context);

    /**
     * 根据资源下载链接进行下载
     *
     * @param context 上下文
     */
    private void downloadRes(CrawResourceContext context) {
        Long timeLimit = context.getDto().getHandlerTimeLimit();

        int curIndex = 0;
        int total = context.getThirdProductEntityList().size();
        Map<String, String> downloadUrlMap = context.getDownloadUrlMap();
        for (ThirdProductEntity entity : context.getThirdProductEntityList()) {
            log.info("爬取资源：下载进度 {}/{}", ++curIndex, total);
            // 限制下载速度
            TimeUtils.sleep(timeLimit);
            // 下载zip(最多尝试下载3次)
            downloadZip(downloadUrlMap, entity, 3);
        }
    }

    private void downloadZip(Map<String, String> downloadUrlMap, ThirdProductEntity entity, int count) {
        if (count <= 0) {
            log.error("多次下载失败：{}", entity.getThirdResId());
            return;
        }
        RequestCallback requestCallback = request -> request.getHeaders()
                .setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL));
        // 下载文件，对响应进行流式处理而不是将其全部加载到内存中
        restTemplate.execute(downloadUrlMap.get(entity.getThirdResId()), HttpMethod.GET, requestCallback, clientHttpResponse -> {
            // 先删除旧文件再下载
            if (cn.hutool.core.io.FileUtil.exist(entity.getZipPath())) {
                cn.hutool.core.io.FileUtil.del(entity.getZipPath());
            }
            Files.copy(clientHttpResponse.getBody(), Paths.get(entity.getZipPath()));
            return null;
        });

        File file = new File(entity.getZipPath());
        if (!file.exists() || file.length() == 0) {
            downloadZip(downloadUrlMap, entity, --count);
        }
    }

    /**
     * 新增商品和商品分类映射记录
     *
     * @param context 上下文
     */
    @Transactional(rollbackFor = Exception.class)
    void save(CrawResourceContext context) {
        List<ThirdProductEntity> thirdProductEntityList = context.getThirdProductEntityList();

        if (!thirdProductService.saveBatch(thirdProductEntityList)) {
            throw new BusinessException("插入第三方商品记录失败： " + context.getCurUrl());
        }

        Long curThirdCategoryKeyId = context.getThirdResourceCategoryMap().get(context.getCurThirdCategory());
        List<ThirdProductCategoryMappingEntity> thirdProductCategoryMappingEntityList = thirdProductEntityList.stream()
                .map(thirdProductEntity -> {
                    ThirdProductCategoryMappingEntity mappingEntity = new ThirdProductCategoryMappingEntity();
                    mappingEntity.setProductId(thirdProductEntity.getId());
                    mappingEntity.setCategoryId(curThirdCategoryKeyId);
                    mappingEntity.setSource(context.getDto().getSource());
                    return mappingEntity;
                }).collect(Collectors.toList());

        if (!thirdProductCategoryService.saveBatch(thirdProductCategoryMappingEntityList)) {
            throw new BusinessException("插入第三方商品分类记录失败： " + context.getCurUrl());
        }

        log.info("爬取资源入库={}", context.getThirdResIdList());
    }

    /**
     * 流程开始处理前，进行初始化操作
     *
     * @param context 上下文
     */
    protected void init(CrawResourceContext context) {
        CrawlerResourceTaskDTO dto = context.getDto();

        String targetFolder = FileUtil.appendSeparator(dto.getTargetFolder()) + FileUtil.appendSeparator(DateUtils.nowDateForYmd());

        // 全局设置下载路径
        dto.setTargetFolder(targetFolder);

        // 校验并创建目录
        FileUtil.createDirectory(targetFolder);

        // 拉取第三方二级分类，不指定地址，会拉取所有有效的分类地址
        List<ThirdCategoryEntity> thirdResCategoryList = thirdCategoryService.list(new LambdaQueryWrapper<ThirdCategoryEntity>()
                .select(ThirdCategoryEntity::getId, ThirdCategoryEntity::getCategoryId)
                .eq(ThirdCategoryEntity::getSource, dto.getSource())
                .eq(ThirdCategoryEntity::getLevel, 2)
                .eq(CollectionUtils.isEmpty(dto.getUrls()), ThirdCategoryEntity::getStatus, StatusEnum.VALID.getCode())
                .in(CollectionUtils.isNotEmpty(dto.getUrls()), ThirdCategoryEntity::getCategoryId, dto.getUrls())
        );
        Map<String, Long> thirdResCategoryListMap = thirdResCategoryList.stream()
                .collect(Collectors.toMap(ThirdCategoryEntity::getCategoryId, ThirdCategoryEntity::getId));
        context.setThirdResourceCategoryMap(thirdResCategoryListMap);
    }

    /**
     * 流程处理结束后，进行收尾操作
     *
     * @param context 上下文
     */
    protected void end(CrawResourceContext context) {
    }

    /**
     * 单次循环初始化
     *
     * @param context 上下文
     */
    protected void initLoop(CrawResourceContext context) {
        if (CollectionUtils.isEmpty(context.getUrlList())) {
            context.setSkipProcess(true);
        } else {
            context.setSkipProcess(false);
        }
    }

    /**
     * 单次循环结束操作
     *
     * @param context 上下文
     */
    protected void endLoop(CrawResourceContext context) {
    }
}
